# -*-* encoding:UTF-8 -*-
# author            : mengy
# date              : 2019/6/11
# python-version    : Python 3.7.0
# description       : 微博用户数据分析

import re, os

import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
from pyecharts.charts import Line, Bar, Pie, Map, Geo
from pyecharts.options import global_options as global_opts, series_options as series_opts

from sina_crawl.utils.mongodb import MongoDbClient
from sina_crawl.utils.logger_utils import Logger

log = Logger()

# 设置字体，支持中文
font_set = FontProperties(fname=r"C:\Windows\Fonts\msyh.ttc", size=12)

db = MongoDbClient()

# 使用 pyecharts 绘制图表
GRAPHIC_ECHARTS = "ECHARTS"
# 使用 pyplot 绘制图表
GRAPHIC_PLOT = "PLOT"

PATH = __file__


def line_base(title=u'Line 基本示例') -> Line:
    """
    绘制折线图基本配置\n
    :param title: 图表标题\v
    :return:
    """
    c = (
        Line(init_opts=global_opts.InitOpts(width="100%", height="600px"))
            .set_global_opts(title_opts=global_opts.TitleOpts(title=title), toolbox_opts=global_opts.ToolboxOpts())
    )
    return c


def bar_base(title=u'Bar 基本示例', width="100%", height="600px") -> Bar:
    """
     绘制柱状图基本配置\n
    :param title: 图表标题\n
    :param width: 图表显示宽度\n
    :param height: 图表显示长度\n
    :return:
    """
    c = (
        Bar(init_opts=global_opts.InitOpts(width=width, height=height))
            .set_global_opts(title_opts=global_opts.TitleOpts(title=title),
                             toolbox_opts=global_opts.TitleOpts(),
                             legend_opts=global_opts.LegendOpts(is_show=True))
    )
    return c


def pie_base(title=u'Pie 基本示例') -> Pie:
    """
    绘制饼图基本配置\n
    :param title: 图表标题\n
    :return:
    """
    c = (
        Pie(init_opts=global_opts.InitOpts(width="100%", height="600px"))
            .set_series_opts(label_opts=series_opts.LabelOpts(formatter="{b}"))
            .set_global_opts(title_opts=global_opts.TitleOpts(title=title),
                             toolbox_opts=global_opts.ToolboxOpts())
    )
    return c


def geo_base(title=u'Geo 基本示例') -> Geo:
    """
    绘制地理坐标图基本配置\n
    :param title: 图表标题\n
    :return:
    """
    c = (
        Geo(init_opts=global_opts.InitOpts(width="100%", height="600px"))
            .add_schema(maptype="china")
            .set_series_opts(label_opts=series_opts.LabelOpts(is_show=True))
            .set_global_opts(visualmap_opts=global_opts.VisualMapOpts(),
                             title_opts=global_opts.TitleOpts(title=title),
                             toolbox_opts=global_opts.ToolboxOpts())
    )
    return c


def map_base(title=u'Map 基本示例', **kwargs) -> Map:
    """
    绘制地图基本配置\n
    :param title: 图表标题\n
    :return:
    """
    max_value = kwargs.get('max')
    c = (
        Map(init_opts=global_opts.InitOpts(width="100%", height="600px"))
            .set_series_opts(label_opts=series_opts.LabelOpts(is_show=True))
            .set_global_opts(title_opts=global_opts.TitleOpts(title=title), legend_opts=global_opts.LegendOpts(),
                             toolbox_opts=global_opts.ToolboxOpts(),
                             visualmap_opts=global_opts.VisualMapOpts(max_=max_value))
    )
    return c


def analysis_crawl(graphic=GRAPHIC_PLOT):
    """
    日爬取量统计\n
    :param graphic:绘制工具\n
    :return:
    """
    assert graphic in [GRAPHIC_ECHARTS, GRAPHIC_PLOT], '非法的绘制工具'
    title = u'日爬取量统计图'

    def get_data() -> dict:
        # 对爬虫数据按照 爬取时间(crawl_time) 进行分组统计
        match = {
            "crawl_time": {
                "$nin": [None, 'null', '']
            }
        }
        group_by = 'crawl_time'
        group = {
            "_id": "$%s" % (group_by if group_by else None),
            "count": {"$sum": 1}
        }
        res = db.aggregate([
            {"$match": match},
            {"$group": group},
            # {"$sort": {'crawl_time': -1}}
        ])
        # 由于 db 存储的时间精确到秒，需要重新对数据按照日期进行分组操作
        dic = {}
        for item in res:
            # 提取日期
            key = item.get('_id').split(' ')[0]
            origin = int(dic.get(key)) if dic.get(key) else 0
            value = origin + item.get('count')
            dic.update({key: value})
        log.debug(dic)
        # 排序
        keys = sorted(dic.keys())
        aa = list(map(dic.get, keys))
        return dict(zip(keys, aa))

    dic = get_data()
    if graphic == GRAPHIC_PLOT:
        plt.plot(dic.keys(), dic.values())
        # 横坐标值太长，自动调整
        plt.gcf().autofmt_xdate()
        plt.grid()
        plt.title(title, fontproperties=font_set)
        plt.show()
    else:
        line = line_base(title)
        try:
            xdata = tuple(dic.keys())
            ydata = dic.values()
            line.page_title = title
            line.add_xaxis(xdata)
            line.add_yaxis(u'日爬取记录', ydata, is_smooth=True,
                           # 标记最大值和最小值
                           markpoint_opts=series_opts.MarkPointOpts(
                               data=[series_opts.MarkPointItem(type_="max"), series_opts.MarkPointItem(type_="min")]),
                           # 显示平均线
                           markline_opts=series_opts.MarkLineOpts(data=[series_opts.MarkLineItem(type_="average")]))
            line.set_global_opts(title_opts=global_opts.TitleOpts(title=title), toolbox_opts=global_opts.ToolboxOpts(),
                                 xaxis_opts=global_opts.AxisOpts(name=u'日期',
                                                                 # 设置横坐标显示所有日期，并且旋转逆时针30度
                                                                 axislabel_opts=global_opts.LabelOpts(interval=0,
                                                                                                      rotate=30)),
                                 yaxis_opts=global_opts.AxisOpts(name=u'日爬取量', splitline_opts=global_opts.SplitLineOpts(
                                     is_show=True), is_scale=True))
        except Exception as e:
            log.error(e)
            pass
        path = os.path.join(__file__, '../result/analysis_crawl.html')
        line.render(path)


def analysis_gender(graphic=GRAPHIC_ECHARTS):
    """
    新浪微博用户性别分布统计\n
    :param graphic:绘制工具\n
    :return:
    """
    assert graphic in [GRAPHIC_ECHARTS, GRAPHIC_PLOT], '非法的绘制工具'
    male = db.count({"gender": "男"})
    female = db.count({"gender": "女"})
    other = db.count({"gender": {"$nin": ["男", "女"]}})
    total = db.count({})
    log.debug('男：%s  女：%s   其他：%s   总计：%s' % (male / total, female / total, other / total, total))
    title = u'新浪微博用户性别分布图'
    if GRAPHIC_PLOT == graphic:
        plt.bar([u'Male', u'Female', u'Unknown'], [male / total, female / total, other / total])
        plt.bar('Male', male / total, color='#6DAFFF')
        plt.bar('Female', female / total, color='#FF8AA2')
        plt.bar('Unknown', other / total, color="gray")
        plt.ylabel(u'用户占比（%）', fontproperties=font_set)
        plt.title(title, fontproperties=font_set)
        # plt.legend()
        plt.show()
    else:
        attr = ["男", "女", "未知"]
        data = [male, female, other]
        bar = bar_base(title, width="60%", height="400px")
        try:
            bar.page_title = title
            bar.add_xaxis(xaxis_data=attr)
            bar.add_yaxis(series_name=u'性别', yaxis_data=data, itemstyle_opts=series_opts.ItemStyleOpts(color='#6DAFFF'),
                          category_gap="70%", label_opts=series_opts.LabelOpts(is_show=True, formatter="{b}（{c}）"))
            bar.set_global_opts(title_opts=global_opts.TitleOpts(title=title),
                                yaxis_opts=global_opts.AxisOpts(name=u'人数', splitline_opts=global_opts.SplitLineOpts(
                                    is_show=True), is_scale=True),
                                xaxis_opts=global_opts.AxisOpts(name=u'性别'))
            bar.page_title = title
        except Exception as e:
            log.error(e)
            pass
        path = os.path.join(__file__, '../result/analysis_gender.html')
        bar.render(path)


def analysis_age(graphic=GRAPHIC_PLOT):
    """
    新浪微博用户年龄分布统计\n
    :param graphic: 绘制工具\n
    :return:
    """
    assert graphic in [GRAPHIC_ECHARTS, GRAPHIC_PLOT], '非法的绘制工具'
    title = u'新浪微博用户年龄分布图'

    def date_format(date: str) -> str:
        year = ''
        try:
            re_year = re.compile('(\d+)-\d+-\d+').findall(str(date))
            year = re_year[0] if re_year else ''
        except Exception as e:
            log.error(e)
            pass
        return year

    def get_age_range(years: list) -> tuple:
        y_60, y_70, y_80, y_90, y_00, y_10, y__ = [], [], [], [], [], [], []
        for item in years:
            if item:
                if 1960 <= int(item) <= 1969:
                    y_60.append(item)
                elif 1970 <= int(item) <= 1979:
                    y_70.append(item)
                elif 1980 <= int(item) <= 1989:
                    y_80.append(item)
                elif 1990 <= int(item) <= 1999:
                    y_90.append(item)
                elif 2000 <= int(item) <= 2009:
                    y_00.append(item)
                elif 2010 <= int(item) <= 2019:
                    y_10.append(item)
                else:
                    y__.append(item)
            else:
                y__.append(item)
        return y_60, y_70, y_80, y_90, y_00, y_10, y__

    ages = db.find({'birthday': {'$nin': [None, '', 'null']}}, {"birthday": 1, "_id": 0})
    years = [date_format(item) for item in ages]
    total = len(years)
    age_tuple = get_age_range(years)
    data = list(map(lambda ls: len(ls), age_tuple))
    ratio = list(map(lambda ls: len(ls) / total, age_tuple))
    if graphic == GRAPHIC_PLOT:
        # labels = ('60后', '70后', '80后', '90后', '00后', '10后', '其他')
        labels = ('1960 - 1969', '1970 - 1979', '1980 - 1989', '1990 - 1999', '2000 - 2009', '2010 - 2019', 'Other')
        # colors = ('r',)
        plt.pie(ratio, labels=labels, shadow=True, autopct='%1.1f%%')
        plt.title(title, fontproperties=font_set)
        plt.legend()
        plt.show()
    else:
        # labels = ('60后', '70后', '80后', '90后', '00后', '10后', '其他')
        labels = ('1960 - 1969', '1970 - 1979', '1980 - 1989', '1990 - 1999', '2000 - 2009', '2010 - 2019', 'Other')
        data_pair = list(zip(labels, data))
        log.debug(data_pair)
        pie = pie_base(title)
        try:
            pie.add(series_name=title, data_pair=data_pair, radius=["25%", "60%"])
            pie.set_series_opts(label_opts=series_opts.LabelOpts(formatter="{b}：{c}（{d} %）"))
            pie.page_title = title
        except Exception as e:
            log.error(e)
            pass
        path = os.path.join(__file__, '../result/analysis_age.html')
        pie.render(path)


def analysis_activity(graphic=GRAPHIC_PLOT):
    """
    新浪微博用户活跃度\n
    :param graphic: 绘制工具\n
    :return:
    """
    assert graphic in [GRAPHIC_ECHARTS, GRAPHIC_PLOT], '非法的绘制工具'
    title = u'新浪微博用户活跃度分布图'

    def distinguish(weibo_nums):
        level0, level1, level2, level3, level4, level5 = 0, 0, 0, 0, 0, 0
        for item in weibo_nums:
            if item < 10:
                level0 += 1
            elif 10 <= item < 50:
                level1 += 1
            elif 50 <= item < 200:
                level2 += 1
            elif 200 <= item < 500:
                level3 += 1
            elif 500 <= item < 1000:
                level4 += 1
            else:
                level5 += 1
        return level0, level1, level2, level3, level4, level5

    weibo_nums = db.find({}, {"webo_num": 1, '_id': 0})
    counts = [item['webo_num'] for item in weibo_nums if item]
    slices = distinguish(counts)
    labels = ['[0,10)', '[10,50)', '[50,200)', '[200,500)', '[500,1000)', '1000+']
    if graphic == GRAPHIC_PLOT:
        plt.pie(slices, labels=labels, shadow=True, autopct='%1.11f%%')
        plt.title(title, fontproperties=font_set)
        plt.legend()
        plt.show()
    else:
        pie = pie_base(title)
        data = list(zip(labels, slices))
        try:
            pie.page_title = title
            pie.add(series_name=u'微博发布数量', data_pair=data, rosetype='area', radius=["20%", "60%"])
            pie.set_series_opts(label_opts=series_opts.LabelOpts(formatter="{b}：{c}（{d} %）"))
        except Exception as e:
            log.error(e)
            pass
        path = os.path.join(__file__, '../result/analysis_activity.html')
        pie.render(path)


def analysis_fans_follows(graphic=GRAPHIC_PLOT):
    """
    新浪微博用户粉丝和关注数分布\n
    :param graphic: 绘制工具\n
    :return:
    """
    assert graphic in [GRAPHIC_ECHARTS, GRAPHIC_PLOT], '非法的绘制工具'
    title = u'新浪微博用户粉丝和关注数分布图'

    def distinguish_fans(list: list) -> tuple:
        level0, level1, level2, level3, level4, level5, level6, level7 = 0, 0, 0, 0, 0, 0, 0, 0
        for item in list:
            if isinstance(item, str):
                item = int(item)
            if item:
                if item < 20:
                    level0 += 1
                elif 20 <= item < 200:
                    level1 += 1
                elif 200 <= item < 1000:
                    level2 += 1
                elif 1000 <= item < 10000:
                    level3 += 1
                elif 10000 <= item < 100000:
                    level4 += 1
                elif 100000 <= item < 1000000:
                    level5 += 1
                elif 1000000 <= item < 10000000:
                    level6 += 1
                else:
                    level7 += 1
        return level0, level1, level2, level3, level4, level5, level6, level7

    def distinguish_follows(list: list) -> tuple:
        level0, level1, level2, level3, level4 = 0, 0, 0, 0, 0
        for item in list:
            if item:
                if item < 20:
                    level0 += 1
                elif 20 <= item < 200:
                    level1 += 1
                elif 200 <= item < 1000:
                    level2 += 1
                elif 1000 <= item < 10000:
                    level3 += 1
                else:
                    level4 += 1
        return level0, level1, level2, level3, level4

    users = db.find({}, {"fans_num": 1, "follow_num": 1, "_id": 0})

    def get_fans_follow(user):
        return user.get("fans_num"), user.get("follow_num")

    fans_follows_cnt = [get_fans_follow(user) for user in users]
    fans_cnt = [item[0] for item in fans_follows_cnt if item]
    follows_cnt = [item[1] for item in fans_follows_cnt if item]
    slice1 = distinguish_fans(fans_cnt)
    slice2 = distinguish_follows(follows_cnt)
    labels1 = ['[0,20)', '[20,200)', '[200,1000)', '[1000,1w)', '[1w,10w)', '[10w,100w)',
               '[100w,1000w)', '1000w+']
    labels2 = ['[0,20)', '[20,200)', '[200,1000)', '[1000,1w)', '1w+']
    if graphic == GRAPHIC_PLOT:
        # 设置画布的大小（11，4）代表分辨率为： 1100 × 400
        plt.figure(1, figsize=(11, 4))
        plt.subplot(131)
        plt.pie(slice1, labels=labels1, shadow=True, autopct='%1.1f%%')
        plt.title(u'粉丝', fontproperties=font_set)
        plt.subplot(132)
        plt.title(u'关注', fontproperties=font_set)
        plt.pie(slice2, labels=labels2, shadow=True, autopct='%1.1f%%')
        plt.suptitle(title, fontproperties=font_set)
        plt.show()
    else:
        pie = pie_base(title)
        data1 = list(zip(labels1, slice1))
        data2 = list(zip(labels2, slice2))
        try:
            pie.page_title = title
            pie.add(series_name=u'用户粉丝统计', data_pair=data1, radius=["10%", "30%"], center=["20%", "50%"])
            pie.add(series_name=u'用户关注统计', data_pair=data2, radius=["10%", "30%"], center=["60%", "50%"])
            pie.set_series_opts(label_opts=series_opts.LabelOpts(formatter="{b}：{c}（{d} %）"))
            pie.set_global_opts(title_opts=global_opts.TitleOpts(title=title),
                                legend_opts=global_opts.LegendOpts(type_="scroll", pos_bottom="30%", pos_right="5%",
                                                                   orient="vertical"),
                                toolbox_opts=global_opts.ToolboxOpts())
        except Exception as e:
            log.error(e)
            pass
        path = os.path.join(__file__, '../result/analysis_fans_follows.html')
        pie.render(path)


def analysis_district() -> None:
    """
    新浪微博用户地区分布图\n
    :return:
    """

    def get_data():
        match = {
            'district': {
                '$nin': ['null', '']
            }
        }
        # 分组的关键字
        groupby = 'province'

        group = {
            '_id': "$%s" % (groupby if groupby else None),
            'count': {'$sum': 1}
        }
        res = db.aggregate([
            # {'$match': match},
            {'$group': group},
        ])
        result = [(re.get('_id'), re.get('count')) for re in res]
        log.debug(result)
        return result

    title = u'新浪微博用户分布地图'
    # geo = geo_base(title)
    data = get_data()
    max_district, max_value = '', 0
    d, other = [], 0
    for tup in data:
        if max_value < tup[1]:
            max_district, max_value = tup
        if not tup[0] or tup[0] == '' or tup[0] == '其他' or tup[0] == '海外':
            other += tup[1]
        else:
            d.append(tup)
    d.append(('其他', other))
    print(max_district, max_value)
    map = map_base(title, max=max_value)
    try:
        # ratio = 500 / db.count({'province': {'$nin': ['null', '', '其他', ' ', '海外']}})
        # data = [('四川', 1269 * ratio), ('吉林', 481 * ratio), ('香港', 385 * ratio), ('广东', 3051 * ratio),
        #         ('山东', 1267 * ratio), ('安徽', 624 * ratio), ('辽宁', 760 * ratio), ('贵州', 1012 * ratio),
        #         ('宁夏', 245 * ratio), ('江苏', 1457 * ratio), ('台湾', 355 * ratio), ('北京', 5090 * ratio),
        #         ('重庆', 867 * ratio), ('山西', 496 * ratio), ('内蒙古', 391 * ratio), ('青海', 265 * ratio),
        #         ('湖南', 1098 * ratio), ('浙江', 1893 * ratio), ('新疆', 366 * ratio), ('甘肃', 317 * ratio),
        #         ('澳门', 90 * ratio), ('上海', 1856 * ratio), ('河北', 742 * ratio), ('福建', 941 * ratio), ('天津', 487 * ratio),
        #         ('湖北', 862 * ratio), ('陕西', 662 * ratio), ('云南', 478 * ratio), ('海南', 327 * ratio), ('西藏', 228 * ratio),
        #         ('河南', 895 * ratio), ('广西', 503 * ratio), ('黑龙江', 485 * ratio), ('江西', 476 * ratio)]
        # geo.add_schema(maptype='china', itemstyle_opts=series_opts.ItemStyleOpts(color='#ffffff'))
        # geo.page_title = title
        # geo.add("", data, type_='effectScatter', is_large=True)
        map.page_title = title
        map.add(series_name=title, data_pair=d, maptype='china', tooltip_opts=global_opts.ToolboxOpts())
    except Exception as e:
        log.error(e)
        pass
    # geo.render('result/analysis_district.html')
    path = os.path.join(__file__, '../result/analysis_district.html')
    map.render(path)


def freash_analysis():
    """
    刷新统计结果
    :return:
    """
    analysis_crawl(graphic=GRAPHIC_ECHARTS)
    analysis_gender(graphic=GRAPHIC_ECHARTS)
    analysis_age(graphic=GRAPHIC_ECHARTS)
    analysis_activity(graphic=GRAPHIC_ECHARTS)
    analysis_fans_follows(graphic=GRAPHIC_ECHARTS)
    analysis_district()


if __name__ == '__main__':
    freash_analysis()
